ostbuild: Re-unify into one snapshot.json file
authorColin Walters <walters@verbum.org>
Tue, 13 Mar 2012 14:13:22 +0000 (10:13 -0400)
committerColin Walters <walters@verbum.org>
Tue, 13 Mar 2012 14:39:31 +0000 (10:39 -0400)
This is actually easier.  Now add 'ostbuild query-content' which can
extract the JSON data we stuff into builds.

Makefile-ostbuild.am
src/ostbuild/pyostbuild/builtin_build.py
src/ostbuild/pyostbuild/builtin_checkout.py
src/ostbuild/pyostbuild/builtin_chroot_compile_one.py
src/ostbuild/pyostbuild/builtin_gen_snapshot.py [deleted file]
src/ostbuild/pyostbuild/builtin_query_content.py [new file with mode: 0755]
src/ostbuild/pyostbuild/builtin_resolve.py
src/ostbuild/pyostbuild/builtins.py
src/ostbuild/pyostbuild/main.py

index cc5ed5ecd45af329e5bc0de789e600aad6698d16..08eb8c034928d57435fd98584443ee32923400f1 100644 (file)
@@ -27,7 +27,7 @@ pyostbuild_PYTHON =                                   \
        src/ostbuild/pyostbuild/builtin_chroot_compile_one.py   \
        src/ostbuild/pyostbuild/builtin_chroot_run_triggers.py  \
        src/ostbuild/pyostbuild/builtin_compile_one.py  \
-       src/ostbuild/pyostbuild/builtin_gen_snapshot.py \
+       src/ostbuild/pyostbuild/builtin_query_content.py        \
        src/ostbuild/pyostbuild/builtin_resolve.py      \
        src/ostbuild/pyostbuild/builtin_status.py       \
        src/ostbuild/pyostbuild/builtins.py             \
index 88073bb651b28d54dc7cf5cd16649e4376a2d9d0..3603932798e8dff8aa4667933a4ec14ecedf798a 100755 (executable)
@@ -60,38 +60,6 @@ class OstbuildBuild(builtins.Builtin):
         run_sync(args, cwd=cwd, fatal_on_error=False, keep_stdin=True)
         fatal("Exiting after debug shell")
 
-    def _resolve_component_meta(self, component_meta):
-        result = dict(component_meta)
-        orig_src = component_meta['src']
-
-        did_expand = False
-        for (vcsprefix, expansion) in self.manifest['vcsconfig'].iteritems():
-            prefix = vcsprefix + ':'
-            if orig_src.startswith(prefix):
-                result['src'] = expansion + orig_src[len(prefix):]
-                did_expand = True
-                break
-
-        name = component_meta.get('name')
-        if name is None:
-            if did_expand:
-                src = orig_src
-                idx = src.rindex(':')
-                name = src[idx+1:]
-            else:
-                src = result['src']
-                idx = src.rindex('/')
-                name = src[idx+1:]
-            if name.endswith('.git'):
-                name = name[:-4]
-            name = name.replace('/', '-')
-            result['name'] = name
-
-        if 'branch' not in result:
-            result['branch'] = 'master'
-
-        return result
-
     def _build_one_component(self, name, component):
         branch = component['branch']
         architecture = component['architecture']
@@ -226,7 +194,7 @@ class OstbuildBuild(builtins.Builtin):
                           '--union', '--subpath=' + subtree,
                           branch_rev, compose_rootdir])
 
-        contents_path = os.path.join(compose_rootdir, 'manifest.json')
+        contents_path = os.path.join(compose_rootdir, 'contents.json')
         f = open(contents_path, 'w')
         json.dump(metadata, f, indent=4, sort_keys=True)
         f.close()
@@ -249,7 +217,7 @@ class OstbuildBuild(builtins.Builtin):
         self.args = args
         
         self.parse_config()
-        self.parse_components_and_targets()
+        self.parse_snapshot()
 
         self.buildopts = BuildOptions()
         self.buildopts.shell_on_failure = args.shell_on_failure
@@ -259,7 +227,7 @@ class OstbuildBuild(builtins.Builtin):
         if args.recompose:
             pass
         elif len(args.components) == 0:
-            tsorted = buildutil.tsort_components(self.components, 'build-depends')
+            tsorted = buildutil.tsort_components(self.snapshot['components'], 'build-depends')
             tsorted.reverse()
             build_component_order = tsorted
         else:
@@ -267,7 +235,7 @@ class OstbuildBuild(builtins.Builtin):
                 fatal("Can't specify --start-at with component list")
             for name in args.components:
                 found = False
-                component = self.components.get(name)
+                component = self.snapshot['components'].get(name)
                 if component is None:
                     fatal("Unknown component %r" % (name, ))
                 build_component_order.append(name)
@@ -284,10 +252,10 @@ class OstbuildBuild(builtins.Builtin):
             start_at_index = 0
 
         for component_name in build_component_order[start_at_index:]:
-            component = self.components.get(component_name)
+            component = self.snapshot['components'].get(component_name)
             self._build_one_component(component_name, component)
 
-        for target in self.targets['targets']:
+        for target in self.snapshot['targets']:
             self._compose(target)
         
 builtins.register(OstbuildBuild)
index b768fe6273ba3ab46765ca073c9fc905a2539758..553e505ff2337638dcf7cb11a84e2a0f1c8f5f99 100755 (executable)
@@ -47,7 +47,7 @@ class OstbuildCheckout(builtins.Builtin):
         self.args = args
         
         self.parse_config()
-        self.parse_components_and_targets()
+        self.parse_snapshot()
 
         if len(args.components) > 0:
             checkout_components = args.components
@@ -56,7 +56,7 @@ class OstbuildCheckout(builtins.Builtin):
 
         for component_name in checkout_components:
             found = False
-            component = self.components.get(component_name)
+            component = self.snapshot['components'].get(component_name)
             if component is None:
                 fatal("Unknown component %r" % (component_name, ))
             (keytype, uri) = buildutil.parse_src_key(component['src'])
index a77d6cfc8dda8795387917a46e88d3dbf8c1a659..40d46652f66993ecea11b31e7b665319379d58e3 100755 (executable)
@@ -31,10 +31,11 @@ class OstbuildChrootCompileOne(builtins.Builtin):
     short_description = "Build artifacts from the current source directory in a chroot"
 
     def _compose_buildroot(self, component_name, dirpath):
-        dependencies = buildutil.build_depends(component_name, self.components)
-        component = self.components.get(component_name)
+        components = self.snapshot['components']
+        dependencies = buildutil.build_depends(component_name, components)
+        component = components.get(component_name)
 
-        base_devel_name = 'bases/%s-%s-%s' % (self.manifest['base-prefix'],
+        base_devel_name = 'bases/%s-%s-%s' % (self.snapshot['base-prefix'],
                                               component['architecture'],
                                               'devel')
         checkout_trees = [(base_devel_name, '/')]
@@ -58,7 +59,7 @@ class OstbuildChrootCompileOne(builtins.Builtin):
         args = parser.parse_args(argv)
 
         self.parse_config()
-        self.parse_components_and_targets()
+        self.parse_snapshot()
 
         if args.name:
             component_name = args.name
@@ -68,7 +69,8 @@ class OstbuildChrootCompileOne(builtins.Builtin):
             parentparent = os.path.dirname(parent)
             component_name = '%s/%s/%s' % tuple(map(os.path.basename, [parentparent, parent, cwd]))
 
-        component = self.components.get(component_name)
+        components = self.snapshot['components']
+        component = components.get(component_name)
         if component is None:
             fatal("Couldn't find component '%s' in manifest" % (component_name, ))
         self.metadata = dict(component)
diff --git a/src/ostbuild/pyostbuild/builtin_gen_snapshot.py b/src/ostbuild/pyostbuild/builtin_gen_snapshot.py
deleted file mode 100755 (executable)
index c70b806..0000000
+++ /dev/null
@@ -1,68 +0,0 @@
-# Copyright (C) 2011,2012 Colin Walters <walters@verbum.org>
-#
-# This library is free software; you can redistribute it and/or
-# modify it under the terms of the GNU Lesser General Public
-# License as published by the Free Software Foundation; either
-# version 2 of the License, or (at your option) any later version.
-#
-# This library is distributed in the hope that it will be useful,
-# but WITHOUT ANY WARRANTY; without even the implied warranty of
-# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
-# Lesser General Public License for more details.
-#
-# You should have received a copy of the GNU Lesser General Public
-# License along with this library; if not, write to the
-# Free Software Foundation, Inc., 59 Temple Place - Suite 330,
-# Boston, MA 02111-1307, USA.
-
-# ostbuild-compile-one-make wraps systems that implement the GNOME build API:
-# http://people.gnome.org/~walters/docs/build-api.txt
-
-import os,sys,stat,subprocess,tempfile,re,shutil
-import argparse
-from StringIO import StringIO
-import json
-
-from . import builtins
-from .ostbuildlog import log, fatal
-from .subprocess_helpers import run_sync, run_sync_get_output
-from . import buildutil
-
-class OstbuildGenSnapshot(builtins.Builtin):
-    name = "gen-snapshot"
-    short_description = "Generate a snapshot description from a tree"
-
-    def __init__(self):
-        builtins.Builtin.__init__(self)
-
-    def execute(self, argv):
-        parser = argparse.ArgumentParser(description=self.short_description)
-        parser.add_argument('--branch', required=True)
-
-        args = parser.parse_args(argv)
-        self.args = args
-        self.parse_config()
-
-        contents_json_text = run_sync_get_output(['ostree', '--repo=' + self.repo,
-                                                  'cat', args.branch, 'contents.json'])
-        
-        contents = json.loads(contents_json_text)
-        contents_list = contents['contents']
-
-        base = contents_list[0]
-        artifacts = contents_list[1:]
-
-        components = []
-        snapshot = {'name': args.branch,
-                    'base': contents['base'],
-                    'components': components}
-
-        for artifact in artifacts:
-            component_meta_text = run_sync_get_output(['ostree', '--repo=' + self.repo,
-                                                       'cat', artifact['rev'], '/_ostbuild-meta.json'])
-            component_meta = json.loads(component_meta_text)
-            components.append(component_meta)
-
-        json.dump(snapshot, sys.stdout)
-    
-builtins.register(OstbuildGenSnapshot)
diff --git a/src/ostbuild/pyostbuild/builtin_query_content.py b/src/ostbuild/pyostbuild/builtin_query_content.py
new file mode 100755 (executable)
index 0000000..89f5839
--- /dev/null
@@ -0,0 +1,67 @@
+# Copyright (C) 2011,2012 Colin Walters <walters@verbum.org>
+#
+# This library is free software; you can redistribute it and/or
+# modify it under the terms of the GNU Lesser General Public
+# License as published by the Free Software Foundation; either
+# version 2 of the License, or (at your option) any later version.
+#
+# This library is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+# Lesser General Public License for more details.
+#
+# You should have received a copy of the GNU Lesser General Public
+# License along with this library; if not, write to the
+# Free Software Foundation, Inc., 59 Temple Place - Suite 330,
+# Boston, MA 02111-1307, USA.
+
+# ostbuild-compile-one-make wraps systems that implement the GNOME build API:
+# http://people.gnome.org/~walters/docs/build-api.txt
+
+import os,sys,stat,subprocess,tempfile,re,shutil
+import argparse
+from StringIO import StringIO
+import json
+
+from . import builtins
+from .ostbuildlog import log, fatal
+from .subprocess_helpers import run_sync, run_sync_get_output
+from . import buildutil
+
+class OstbuildQueryContent(builtins.Builtin):
+    name = "query-content"
+    short_description = "Output metadata from a component"
+
+    def __init__(self):
+        builtins.Builtin.__init__(self)
+
+    def execute(self, argv):
+        parser = argparse.ArgumentParser(description=self.short_description)
+        parser.add_argument('--branch', required=True)
+        parser.add_argument('--component')
+
+        args = parser.parse_args(argv)
+        self.args = args
+        self.parse_config()
+
+        contents_json_text = run_sync_get_output(['ostree', '--repo=' + self.repo,
+                                                  'cat', args.branch, 'contents.json'])
+        
+        if args.component is None:
+            sys.stdout.write(contents_json_text)
+        else:
+            contents = json.loads(contents_json_text)
+            contents_list = contents['contents']
+            found = False
+            for content in contents_list:
+                if content['name'] != args.component:
+                    found = True
+                    break
+            if not found:
+                fatal("Unknown component '%s'" % (args.component, ))
+            ostbuildmeta_json = run_sync_get_output(['ostree', '--repo=' + self.repo,
+                                                     'cat', content['ostree-revision'],
+                                                     '/_ostbuild-meta.json'])
+            sys.stdout.write(ostbuildmeta_json)
+
+builtins.register(OstbuildQueryContent)
index 6defbae4af184c3cf7d2e43693edf0aaa2dccacf..3486396d01147363c0e746fa4911ef78b6f0ba0e 100755 (executable)
@@ -193,9 +193,7 @@ class OstbuildResolve(builtins.Builtin):
                 component['patches']['files'] = patch_files
 
         name_prefix = snapshot['name-prefix']
-        del snapshot['name-prefix']
         base_prefix = snapshot['base-prefix']
-        del snapshot['base-prefix']
 
         manifest_architectures = snapshot['architectures']
 
@@ -247,9 +245,8 @@ class OstbuildResolve(builtins.Builtin):
         del snapshot['patches']
         del snapshot['architectures']
 
-        targets_json = {}
         targets_list = []
-        targets_json['targets'] = targets_list
+        snapshot['targets'] = targets_list
         for architecture in manifest_architectures:
             for target_component_type in ['runtime', 'devel']:
                 target = {}
@@ -276,18 +273,15 @@ class OstbuildResolve(builtins.Builtin):
                         component_ref['trees'] = ['/runtime', '/devel', '/doc']
                     contents.append(component_ref)
                 target['contents'] = contents
-        out_targets = os.path.join(self.workdir, '%s-targets.json' % (name_prefix, ))
-        f = open(out_targets, 'w')
-        json.dump(targets_json, f, indent=4, sort_keys=True)
-        f.close()
-        print "Created: %s" % (out_targets, )
 
-        out_components = os.path.join(self.workdir, '%s-components.json' % (name_prefix, ))
-        f = open(out_components, 'w')
         for component in components_by_name.itervalues():
             del component['name']
-        json.dump(components_by_name, f, indent=4, sort_keys=True)
+        snapshot['components'] = components_by_name
+
+        out_snapshot = os.path.join(self.workdir, '%s-snapshot.json' % (name_prefix, ))
+        f = open(out_snapshot, 'w')
+        json.dump(snapshot, f, indent=4, sort_keys=True)
         f.close()
-        print "Created: %s" % (out_components, )
+        print "Created: %s" % (out_snapshot, )
         
 builtins.register(OstbuildResolve)
index 60f5278185c3875ec04da0db17c1d1d69b75aaee..64f7be4eaedec645e3966e8f2652ae3090075572 100755 (executable)
@@ -47,12 +47,10 @@ class Builtin(object):
         self.manifest = json.load(open(self.manifest_path))
         self.name_prefix = self.manifest['name-prefix']
 
-    def parse_components_and_targets(self):
+    def parse_snapshot(self):
         self.parse_manifest()
-        components_path = os.path.join(self.workdir, '%s-components.json' % (self.name_prefix, ))
-        self.components = json.load(open(components_path))
-        targets_path = os.path.join(self.workdir, '%s-targets.json' % (self.name_prefix, ))
-        self.targets = json.load(open(targets_path))
+        snapshot_path = os.path.join(self.workdir, '%s-snapshot.json' % (self.name_prefix, ))
+        self.snapshot = json.load(open(snapshot_path))
 
     def execute(self, args):
         raise NotImplementedError()
index b34d2e6e60283988af9efdc9d4f98cbfb8c1bb0c..842cb5bc4aa60eafa1741f417bec8bcc0a782a90 100755 (executable)
@@ -27,7 +27,7 @@ from . import builtin_checkout
 from . import builtin_chroot_compile_one
 from . import builtin_chroot_run_triggers
 from . import builtin_compile_one
-from . import builtin_gen_snapshot
+from . import builtin_query_content
 from . import builtin_resolve
 from . import builtin_status